/*
* This file is a part of the open-eBackup project.
* This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.
* If a copy of the MPL was not distributed with this file, You can obtain one at
* http://mozilla.org/MPL/2.0/.
*
* Copyright (c) [2024] Huawei Technologies Co.,Ltd.
*
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
*/
#include "FusionStorageVolumeHandler.h"
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <boost/bind.hpp>
#include <boost/algorithm/string.hpp>
#include "FusionStorageDef.h"
#include "volume_handlers/oceanstor/DiskScannerHandler.h"
#include "common/Structs.h"
#include "common/JsonHelper.h"
#include "log/Log.h"
#include "system/System.hpp"
#include "FusionStorageRestApiErrorCode.h"
#include "FusionStorageIscsiDiskScanner.h"
#include "common/utils/Utils.h"

namespace {
    const std::string MODULE_NAME = "FusionStorageVolume";
    const std::string INTERNAL_BACKUP_SCENCE = "1";
    const std::string REG_FS = VirtPlugin::VIRT_PLUGIN_PATH + "bin/reg_fs.sh";
    const std::string DSWAREAGENT_PATH = "/opt/dsware/agent/bin/dsware_agent";
    const std::string DSWAREAGENT_PROCESS_NAME = "./dsware_agent";
    const int RETRY_TIMES = 3;
    const int RETRY_WAITING_TIME = 30;
    const std::string STATUS_IDLE = "idle";
    constexpr uint64_t MOUNT_POINT_EXPIRE_TIME = 24 * 60 * 60;  // 24小时挂载点过期
    const std::string DSWARE_HANDLE_DETACH_DELETE_BITMAP_FAILED = "DETACH_DELETE_BITMAP_FAILED";
    constexpr int64_t INVOKE_API_FAILED_GENERAL_CODE = 1577213460;
    const int32_t MIN_PACIFIC_VERSION_INCREASE = 813;  // 支持使用rest接口创建差量卷的最小pacific版本
    const int GB = 1024 * 1024 * 1024;
    const int32_t NO_STORAGE_INFO = 1577213586;
    const std::string DSWARE_HANDLE_DETACH_BITMAP_FAILED = "DETACH_BITMAP_FAILED";
    const std::string DSWARE_HANDLE_DELETE_BITMAP_FAILED = "DELETE_BITMAP_FAILED";
}  // namespace

VIRT_PLUGIN_NAMESPACE_BEGIN

using ErrorCode = FusionStorageRestApiErrorCode;

FusionStorageVolumeHandler::FusionStorageVolumeHandler(std::shared_ptr<JobHandle> jobHandle, const VolInfo &volInfo,
    std::string jobId, std::string subJobId)
    : VolumeHandler(jobHandle, volInfo, jobId, subJobId), m_isBackup(true), m_dataPersistence(nullptr)
{
    m_apiMode = Module::ConfigReader::getString("HcsConfig", "FusionStorageApiMode");
    std::string deployScence;
    int32_t ret = Utils::GetAgentDeployScence(deployScence);
    if (ret == SUCCESS && deployScence == INTERNAL_BACKUP_SCENCE) {
        m_apiMode = "ISCSI";
    }
    if (ret != SUCCESS) {
        WARNLOG("Failed to get the agent deploy scence.");
    }
    INFOLOG("The api mode is %s.", m_apiMode.c_str());

    if (m_jobHandle != nullptr) {
        m_isBackup = ((m_jobHandle->GetJobType() == JobType::BACKUP) ? true : false);
        m_backupType = m_jobHandle->GetBackupType();
    }
}

FusionStorageVolumeHandler::~FusionStorageVolumeHandler()
{
    if (m_dataPersistence != nullptr) {
        m_dataPersistence->Finish();
    }
}

int32_t FusionStorageVolumeHandler::InitializeVolumeInfo()
{
    m_volSizeInBytes = m_volInfo.m_volSizeInBytes;  // 卷大小

    int32_t iRet;
    if (m_isBackup) {
        iRet = ParseBackupParams();
    } else {
        iRet = ParseRestoreParams();
    }

    if (iRet != SUCCESS) {
        ERRLOG("Params error. fusionStorMgrIp=[%s], poolID=[%s].", m_fusionStorMgrIp.c_str(), m_poolID.c_str());
        return FAILED;
    }

    return InitComponents();
}

int32_t FusionStorageVolumeHandler::ParseBackupParams()
{
    m_fusionStorMgrIp = m_volInfo.m_datastore.m_ip;
    m_poolID = m_volInfo.m_datastore.m_poolId;

    if (m_fusionStorMgrIp == "" || m_poolID == "") {
        return FAILED;
    }

    INFOLOG("Parse backup params succ. fusionStorMgrIp=[%s], poolID=[%s].",
            m_fusionStorMgrIp.c_str(), m_poolID.c_str());
    return SUCCESS;
}

int32_t FusionStorageVolumeHandler::ParseRestoreParams()
{
    m_fusionStorMgrIp = m_volInfo.m_datastore.m_ip;
    m_poolID = m_volInfo.m_datastore.m_poolId;
    m_volumeID = m_volInfo.m_datastore.m_volumeName;

    if (m_fusionStorMgrIp == "" || m_poolID == "" || m_volumeID == "") {
        return FAILED;
    }

    INFOLOG("Parse restore params succ. fusionStorMgrIp=[%s], poolID=[%s], volumeId=[%s].",
            m_fusionStorMgrIp.c_str(), m_poolID.c_str(), m_volumeID.c_str());
    return SUCCESS;
}

int32_t FusionStorageVolumeHandler::InitComponents()
{
    try {
        InitRepoHandler();
        m_dataPersistence = std::make_shared<DiskDataPersistence>();
        if (m_apiMode == "VBS") {
            m_spDSWareApi = std::make_shared<FusionStorageApi>(m_fusionStorMgrIp, m_poolID);
        }
        m_spDSWareRestApi = std::make_shared<FusionStorageRestApiOperator>(m_fusionStorMgrIp, m_poolID,
            m_jobId, m_subJobId);
        m_spDSWareRestApi->SetOpService(m_isOpService);
        m_spDSWareRestApi->InitRestApiOperator();
        if (m_jobHandle != nullptr) {
            int32_t ret = m_spDSWareRestApi->GetStorageInfo(m_jobHandle->GetAppEnv().auth.extendInfo,
                m_volInfo, m_isBackup);
            if (m_apiMode == "ISCSI" && ret != SUCCESS) {
                ERRLOG("Get storage info failed.");
                return ret;
            }
        }

        m_spBitmapHandle = std::make_shared<FusionStorageBitmapHandle>(
                m_fusionStorMgrIp, m_poolID, m_isBackup, m_volInfo.m_uuid, m_dataPersistence,
                m_cacheRepoHandler, m_cacheRepoPath);
        m_spBitmapHandle->InitializeFusionStorageApi(m_apiMode, m_spDSWareRestApi, m_spDSWareApi);

        m_spDeviceFile = std::make_shared<DiskDeviceFile>();
        m_cleanFile = std::make_shared<FusionStorageCleanFile>();
    } catch (std::exception &e) {
        ERRLOG("Init components failed. Exception is %s.", e.what());
        return FAILED;
    }

    INFOLOG("Init fusionstorage handler success.");
    return SUCCESS;
}

bool FusionStorageVolumeHandler::CheckAgentRunning()
{
    std::string filterName = "dsware_agent";
    std::string processName = DSWAREAGENT_PROCESS_NAME;
    std::vector<std::string> resultvec;

    std::vector<Module::CmdParam> cmdParam {
        Module::CmdParam(Module::COMMON_CMD_NAME, "ps"),
        Module::CmdParam(Module::CMD_OPTION_PARAM, "-ef"),
        Module::CmdParam(Module::PIPELINE_PARAM, "|"),
        Module::CmdParam(Module::COMMON_CMD_NAME, "grep"),
        Module::CmdParam(Module::CMD_OPTION_PARAM, "-E"),
        Module::CmdParam(Module::COMMON_PARAM, filterName),
        Module::CmdParam(Module::PIPELINE_PARAM, "|"),
        Module::CmdParam(Module::COMMON_CMD_NAME, "grep"),
        Module::CmdParam(Module::CMD_OPTION_PARAM, "-E"),
        Module::CmdParam(Module::CMD_OPTION_PARAM, "-v"),
        Module::CmdParam(Module::COMMON_CMD_NAME, "grep")
    };
    int runResult = Utils::CallAgentExecCmd(cmdParam, resultvec);
    if (runResult != 0) {
        for (const std::string &result : resultvec) {
            ERRLOG("Cmd failed: %s.", result.c_str());
        }
        ERRLOG("Run cmd failed, filter name: %s, value: %d.", filterName.c_str(), runResult);
        return false;
    }

    for (const std::string &result : resultvec) {
        DBGLOG("OutputList: %s.", result.c_str());
        auto pos = result.find(processName);
        if (pos != std::string::npos) {
            return true;
        }
    }
    ERRLOG("%s is not running.", processName.c_str());
    return false;
}

int32_t FusionStorageVolumeHandler::TestDeviceConnection(const std::string &authExtendInfo, int32_t &erroCode)
{
    std::string errDesc;
    if (m_apiMode == "VBS") {
        if (!CheckHealthStatus(errDesc, erroCode)) {
            ERRLOG("Fusion storage connection failed, error desc: %s.", errDesc.c_str());
            return FAILED;
        }
    } else {
        m_spDSWareRestApi->GetStorageInfo(authExtendInfo, m_volInfo, m_isBackup);
        if (m_spDSWareRestApi->TryGetToken() != SUCCESS) {
            ERRLOG("Fusion storage connection failed, can not get response from server.");
            return FAILED;
        }
        m_spDSWareRestApi->DeleteCurrentSession();
    }

    return SUCCESS;
}

bool FusionStorageVolumeHandler::CheckHealthStatus(std::string &errDesc, int32_t &erroCode)
{
    // check dsware_agent exist
    errDesc.clear();
    bool exists = false;
    try {
        exists = boost::filesystem::exists(DSWAREAGENT_PATH);
    } catch (const boost::filesystem::filesystem_error &e) {
        ERRLOG("Boost::filesystem::exists() exeption: %s, path: %s.", e.code().message(), DSWAREAGENT_PATH.c_str());
        errDesc = DSWAREAGENT_PATH + e.code().message();
        return false;
    }

    if (!exists) {
        ERRLOG("%s is not exist.", DSWAREAGENT_PATH.c_str());
        errDesc = DSWAREAGENT_PATH + " is not exist.";
        return false;
    }

    std::string errDes;
    BitmapVolumeInfo bitmapVolumeInfo;
    bitmapVolumeInfo.volName = "volume-1160e808-a89e-4513-a3cb-e4b036c42d7d_1_72345555_NOTEXIST";

    erroCode = DSWARE_API_OK;
    int32_t retryTimes = RETRY_TIMES;
    while (retryTimes-- >= 0) {
        if (CheckAgentRunning()) {
            erroCode = m_spDSWareApi->QueryBitmapVol(bitmapVolumeInfo, errDes);
            if (erroCode == DSWARE_API_ERR_VOLUME_IS_NOT_EXIST) {
                return true;
            }
            WARNLOG("FusionStorage Agent has an exception, errorCode: %d.", erroCode);
        }
        std::this_thread::sleep_for(std::chrono::seconds(RETRY_WAITING_TIME));
        WARNLOG("Need retry times: %d.", retryTimes + 1);
    }
    ERRLOG("FusionStorage health check failed.");
    errDesc = "query not exist volume failed,maybe dsware_agent process is not running,";
    errDesc += "maybe dsware storage plane network has an exception,";
    errDesc += "maybe Block Storage Clients not in Cluster. maybe dsware-api-xxx.jar file is not exist etc.";
    return false;
}

int32_t FusionStorageVolumeHandler::GetDirtyRanges(const VolSnapInfo &preVolSnapshot, const VolSnapInfo &curVolSnapshot,
    DirtyRanges &dirtyRanges, const uint64_t startOffset, uint64_t &endOffset)
{
    INFOLOG("Begin get dirty range. preSnapId[%s], curSnapId[%s], volSize[%lld], isBackup[%d].",
            preVolSnapshot.m_storageSnapId.c_str(), curVolSnapshot.m_storageSnapId.c_str(), m_volSizeInBytes, m_isBackup);
    if (!m_isBackup && !QuerySnapshot(preVolSnapshot.m_storageSnapId)) {
        INFOLOG("Current and last snapshot is null, will do full restore.");
        dirtyRanges.UseFull();
        DirtyRange everything(startOffset, endOffset - startOffset);
        (void)dirtyRanges.AddRange(everything);
        return SUCCESS;
    }
    m_preChangeID = preVolSnapshot.m_storageSnapId;
    m_changeID = curVolSnapshot.m_storageSnapId;  // 卷的快照ID
    uint64_t diskSize = (endOffset - startOffset) > m_volSizeInBytes ? m_volSizeInBytes : endOffset - startOffset;
    DirtyRangesParams params;
    SetDirtyRangesParams(diskSize, startOffset, params);
    std::string errMsg;
    if (m_apiMode == "ISCSI") {
        int32_t version;
        if (m_spDSWareRestApi->GetVersion(version, errMsg) != SUCCESS || version < MIN_PACIFIC_VERSION_INCREASE
            || m_spDSWareRestApi->m_vbsNodeInfo.mNodeIp.empty()) {
            INFOLOG("Current api mode is iscsi, no way to get bitmap, will do full job.");
            dirtyRanges.UseFull();
            DirtyRange everything(startOffset, endOffset - startOffset);
            (void)dirtyRanges.AddRange(everything);
            return SUCCESS;
        }
    }
    if (m_isBackup && m_preChangeID == m_changeID) {
        ERRLOG("The preSnapshot id is the same with curSnapshot. preSnapshot: %s, curSnapShot: %s.",
            m_preChangeID.c_str(), m_changeID.c_str());
        return FAILED;
    }
    BitmapVolumeInfo bitmapVolumeInfo;
    // 除以2，估算每GB等待超时时间为0.5s
    bitmapVolumeInfo.createTimeOut = m_volSizeInBytes / GB / 2;
    bitmapVolumeInfo.isLastBitmap = endOffset == m_volSizeInBytes ? true : false;
    bitmapVolumeInfo.isFirstBitmap = startOffset == 0 ? true : false;
    m_spBitmapHandle->SetBitmapName(params.volumeID, curVolSnapshot);
    int iRet = m_spBitmapHandle->GetDirtyRanges(params, dirtyRanges, errMsg, bitmapVolumeInfo);
    if (errMsg == DSWARE_HANDLE_DETACH_DELETE_BITMAP_FAILED) {
        AddBitmapInfoToSnapShot(bitmapVolumeInfo.volName, m_preChangeID);
        ReportDeleteBitmap(bitmapVolumeInfo.volName);
    }
    if (iRet != SUCCESS) {
        ERRLOG("Get dirty ranges failed.");
        return FAILED;
    }
    INFOLOG("Get dirty ranges success.");
    return SUCCESS;
}

void FusionStorageVolumeHandler::SetDirtyRangesParams(const uint64_t &diskSize, const uint64_t &startOffset,
    DirtyRangesParams &params)
{
    params.SetDiskSize(diskSize);
    params.SetOffset(startOffset);
    params.SetParentVolumeID(m_preChangeID);
    params.SetVolumeID(m_isBackup ? m_changeID : m_volumeID);
}

bool FusionStorageVolumeHandler::QuerySnapshot(const std::string &snapshotName)
{
    std::string errorDes;
    if (m_apiMode == "ISCSI") {
        std::string  returnBody;
        if (m_spDSWareRestApi->QuerySnapshotInfoByName(snapshotName, returnBody, errorDes) == SUCCESS) {
            INFOLOG("Query snapshot %s success.", snapshotName.c_str());
            return true;
        }
    } else {
        if (m_spDSWareApi->QuerySnapshot(snapshotName, errorDes) == SUCCESS) {
            INFOLOG("Query snapshot %s success.", snapshotName.c_str());
            return true;
        }
    }
    ERRLOG("Query snapshot %s failed.", snapshotName.c_str());
    return false;
}

void FusionStorageVolumeHandler::ReportDeleteBitmap(const std::string &bitmapName)
{
    m_reportArgs = {bitmapName};
    m_reportPara = {
    "virtual_plugin_fusionstorage_volume_delete_bitmap_failed_label",
    JobLogLevel::TASK_LOG_WARNING, SubJobStatus::RUNNING, 0, 0, INVOKE_API_FAILED_GENERAL_CODE};
    Utils::ReportJobDetailWithLabel (m_reportPara, m_reportArgs, m_jobId, m_subJobId);
}

// HCS每次删上一次备份任务的快照，本次备份任务的bitmap删除失败会导致上一次的快照删除失败，将本次任务bitmap加入上一次任务的快照信息中
int32_t FusionStorageVolumeHandler::AddBitmapInfoToSnapShot(const std::string &bitmapName,
    const std::string &snapshotId)
{
    InitRepoHandler();
    std::string preSnapFile = m_metaRepoPath + VIRT_PLUGIN_PRE_SNAPSHOT_INFO;
    SnapshotInfo preSnapshotInfo;
    if (Utils::LoadFileToStructWithRetry(m_metaRepoHandler, preSnapFile, preSnapshotInfo) != SUCCESS) {
        ERRLOG("Load pre snapshot info failed.");
        /* Not return FAILED if no pre-snapshot found. */
    }
    for (auto& snapshot : preSnapshotInfo.m_volSnapList) {
        if (snapshot.m_storageSnapId != snapshotId) {
            continue;
        }
        snapshot.m_bitmapName = bitmapName;
        break;
    }
    std::string preSnapshotStr;
    if (!Module::JsonHelper::StructToJsonString(preSnapshotInfo, preSnapshotStr)) {
        ERRLOG("Convert struct to string failed.");
        return FAILED;
    }
    if (Utils::SaveToFileWithRetry(m_metaRepoHandler, preSnapFile, preSnapshotStr) != SUCCESS) {
        ERRLOG("Save presnapshot indo to file: %s failed.", preSnapFile.c_str());
        return FAILED;
    }
    INFOLOG("Save bitmap name to snap info success.");
    return SUCCESS;
}

// for restore
int32_t FusionStorageVolumeHandler::Open(const VolOpenMode &mode)
{
    INFOLOG("Start to open disk for restore. floatIp[%s], poolId[%s], volumeId[%s], volumeSize[%lld].",
            m_fusionStorMgrIp.c_str(), m_poolID.c_str(), m_volumeID.c_str(), m_volSizeInBytes);
    if (mode == VolOpenMode::READ_ONLY) {
        ERRLOG("Open disk for restore can not use read only mode.");
        return FAILED;
    }

    m_isBackup = false;
    std::string errMsg;
    std::string diskDevicePath;
    int32_t iRet;
    m_dataPersistence->AddObject(FusionStorageDef::DSWARE_MGR_IP, m_fusionStorMgrIp);
    m_dataPersistence->AppendArrayElement(FusionStorageDef::DATA_VOLUMES_ATTACHED, m_volumeID);
    if (m_apiMode == "VBS") {
        m_dataPersistence->AddObject(FusionStorageDef::DSWARE_POOL_ID, m_spDSWareApi->GetPoolID());
        iRet = RetryOp(boost::bind(&FusionStorageApi::AttachVolume, m_spDSWareApi, m_volumeID,
            boost::ref(diskDevicePath), boost::ref(errMsg), m_cacheRepoHandler,
            m_cacheRepoPath + VIRT_PLUGIN_CACHE_ROOT), "AttachVolume");
    } else {
        m_dataPersistence->AddObject(FusionStorageDef::DSWARE_POOL_ID, m_spDSWareRestApi->GetPoolID());
        iRet = RetryOp(boost::bind(&FusionStorageRestApiOperator::AttachVolume, m_spDSWareRestApi,
            m_volumeID, boost::ref(diskDevicePath), boost::ref(errMsg)), "AttachVolume");
    }
    if (iRet != DSWARE_API_OK) {
        ERRLOG("Attach volume failed.");
        m_dataPersistence->RemoveArrayElement(FusionStorageDef::DATA_VOLUMES_ATTACHED, m_volumeID);
        return FAILED;
    }

    // 查询卷是否有预留，有预留则设置key值，没有则不需要处理
    if (m_apiMode == "VBS") {
        int iSetKey = RetryOp(boost::bind(&FusionStorageVolumeHandler::RegKeyForVolume, this,
            boost::ref(diskDevicePath)), "RegKeyForVolume");
        if (iSetKey != DSWARE_API_OK) {
            ERRLOG("Reg Key for volume[%s] failed.iSetKey: %d.", m_volumeID.c_str(), iSetKey);
        }
    }
    DiskScannerHandler::GetInstance()->ChangeFilePriviledge(diskDevicePath, mode);
    int32_t owMode = (mode == VolOpenMode::READ_WRITE) ? O_WRONLY : O_RDONLY;
    DISK_DEVICE_RETURN_CODE deviceCode = m_spDeviceFile->Open(diskDevicePath, owMode, m_volSizeInBytes);
    if (deviceCode != DISK_DEVICE_OK) {
        ERRLOG("Open disk device file[%s] failed.", diskDevicePath.c_str());
        return FAILED;
    }
    INFOLOG("Start to open disk for restore success.");
    return SUCCESS;
}

int32_t FusionStorageRestApiOperator::DeleteVolumeAllMappingCaller(const std::string &volume, std::string &errorDes)
{
    return DeleteVolumeAllMapping(volume, errorDes);
}

// for backup
int32_t FusionStorageVolumeHandler::Open(const VolOpenMode &mode, const BackupSubJobInfo &jobInfo)
{
    int32_t ret = SUCCESS;
    TP_START("TP_FusionStorageOpenForBakFailed", 1, &ret);
    TP_END
    if (ret == FAILED) {
        return FAILED;
    }
    VolSnapInfo snapshot = jobInfo.m_curSnapshotInfo;
    INFOLOG("Start to open disk for backup. managerIp[%s], poolId[%s], snapshotId[%s], volumeSize[%lld]",
            m_fusionStorMgrIp.c_str(), m_poolID.c_str(), snapshot.m_storageSnapId.c_str(), m_volSizeInBytes);

    m_isBackup = true;
    m_changeID = snapshot.m_storageSnapId;
    int32_t iRet;
    std::string errMsg;
    std::string diskDevicePath;
    m_dataPersistence->AddObject(FusionStorageDef::DSWARE_MGR_IP, m_fusionStorMgrIp);
    m_dataPersistence->AppendArrayElement(FusionStorageDef::SNAPSHOT_VOLUMES_ATTACHED, snapshot.m_storageSnapId);
    if (m_apiMode == "VBS") {
        RetryOp(boost::bind(&FusionStorageRestApiOperator::DeleteVolumeAllMappingCaller, m_spDSWareRestApi,
            snapshot.m_storageSnapId, boost::ref(errMsg)), "DetachVolume");
        m_dataPersistence->AddObject(FusionStorageDef::DSWARE_POOL_ID, m_spDSWareApi->GetPoolID());
        iRet = RetryOp(boost::bind(&FusionStorageApi::AttachVolume, m_spDSWareApi, snapshot.m_storageSnapId,
            boost::ref(diskDevicePath), boost::ref(errMsg), m_cacheRepoHandler,
            m_cacheRepoPath + VIRT_PLUGIN_CACHE_ROOT), "AttachVolume");
    } else {
        m_dataPersistence->AddObject(FusionStorageDef::DSWARE_POOL_ID, m_spDSWareRestApi->GetPoolID());
        iRet = RetryOp(boost::bind(&FusionStorageRestApiOperator::AttachVolume, m_spDSWareRestApi,
            snapshot.m_storageSnapId, boost::ref(diskDevicePath), boost::ref(errMsg)), "AttachVolume");
        DiskScannerHandler::GetInstance()->ChangeFilePriviledge(diskDevicePath, mode);
    }
    if (iRet != DSWARE_API_OK) {
        ERRLOG("Attach volume failed.");
        m_dataPersistence->RemoveArrayElement(FusionStorageDef::SNAPSHOT_VOLUMES_ATTACHED, snapshot.m_storageSnapId);
        return FAILED;
    }

    int32_t owMode = (mode == VolOpenMode::READ_WRITE) ? O_WRONLY : O_RDONLY;
    DiskScannerHandler::GetInstance()->ChangeFilePriviledge(diskDevicePath, mode);
    DISK_DEVICE_RETURN_CODE deviceCode = m_spDeviceFile->Open(diskDevicePath, owMode, m_volSizeInBytes);
    if (deviceCode != DISK_DEVICE_OK) {
        ERRLOG("Open disk device file[%s] failed.", diskDevicePath.c_str());
        return FAILED;
    }

    INFOLOG("Open disk for backup success.");
    return SUCCESS;
}

int32_t FusionStorageVolumeHandler::Close()
{
    DBGLOG("Begin close disk.");
    int32_t result = SUCCESS;
    std::string errMsg("Internal error.");
    DISK_DEVICE_RETURN_CODE deviceCode = m_spDeviceFile->Close();
    if (deviceCode != DISK_DEVICE_OK) {
        ERRLOG("Close disk device file failed.");
        errMsg = m_spDeviceFile->GetErrString();
        result = FAILED;
    }

    std::string volumeId;
    std::string cacheDataKey;
    if (m_isBackup) {
        cacheDataKey = FusionStorageDef::SNAPSHOT_VOLUMES_ATTACHED;
        volumeId = m_changeID;
    } else {
        cacheDataKey = FusionStorageDef::DATA_VOLUMES_ATTACHED;
        volumeId = m_volumeID;
        if (m_apiMode != "ISCSI") {
            // 清除注册的key
            std::string diskDevicePath = m_spDeviceFile->GetDiskDevicePath();
            int iDeleteKey =
                RetryOp(boost::bind(&FusionStorageVolumeHandler::UnRegKeyForVolume, this, boost::ref(diskDevicePath)),
                    "UnRegKeyForVolume");
            if (iDeleteKey != DSWARE_API_OK) {
                ERRLOG("Unreg key for volume[%s] failed. iDeleteKey: %d.", diskDevicePath.c_str(), iDeleteKey);
            }
        }
    }
    int32_t iRet;
    if (m_apiMode == "VBS") {
        iRet = RetryOp(
            boost::bind(&FusionStorageApi::DetachVolume, m_spDSWareApi, volumeId, boost::ref(errMsg),
            m_cacheRepoHandler, m_cacheRepoPath + VIRT_PLUGIN_CACHE_ROOT), "DetachVolume");
    } else {
        iRet = RetryOp(
            boost::bind(&FusionStorageRestApiOperator::DetachVolume, m_spDSWareRestApi, volumeId, boost::ref(errMsg)),
            "DetachVolume");
    }

    if (iRet != DSWARE_API_OK) {
        ERRLOG("Detach volume failed.");
        result = FAILED;
    } else {
        m_dataPersistence->RemoveArrayElement(cacheDataKey, volumeId);
    }
    DeleteBitmapVolume();
    DBGLOG("Close disk success.");
    return result;
}

int32_t FusionStorageVolumeHandler::DeleteBitmapVolume()
{
    std::string bitmapFile = m_cacheRepoPath + VIRT_PLUGIN_CACHE_ROOT + m_changeID +
        "_bitmap_volume.info";
    if (m_cacheRepoHandler == nullptr) {
        ERRLOG("Cache repohandle is null.");
        return FAILED;
    }
    if (!m_cacheRepoHandler->Exists(bitmapFile)) {
        WARNLOG("Bitmap file does not exist. file path: %s.", bitmapFile.c_str());
        return FAILED;
    }
    BitmapVolumeInfo bitmapVolumeInfo;
    std::string err = "";
    if (Utils::LoadFileToStruct(m_cacheRepoHandler, bitmapFile, bitmapVolumeInfo) != SUCCESS) {
        ERRLOG("Load bitmap volume info failed.");
        return FAILED;
    }
    m_spBitmapHandle->DeleteAndDetachapVolume(bitmapVolumeInfo.volName, err);
    if (err == DSWARE_HANDLE_DELETE_BITMAP_FAILED || err == DSWARE_HANDLE_DETACH_BITMAP_FAILED) {
        AddBitmapInfoToSnapShot(bitmapVolumeInfo.volName, m_preChangeID);
        ReportDeleteBitmap(bitmapVolumeInfo.volName);
        ERRLOG("Delete and detach bitmap %s failed.", bitmapVolumeInfo.volName.c_str());
        return FAILED;
    }
    m_cacheRepoHandler->Remove(bitmapFile);
    return SUCCESS;
}

int32_t FusionStorageVolumeHandler::ReadBlocks(const uint64_t &offsetInBytes, uint64_t &bufSizeInBytes,
    std::shared_ptr<uint8_t[]> &buf, std::shared_ptr<uint8_t[]> &calBuffer, std::shared_ptr<uint8_t[]> &readBuffer)
{
    int32_t ret = SUCCESS;
    TP_START("TP_FusionStorageReadFailed", 1, &ret);
    TP_END
    if (ret == FAILED) {
        return FAILED;
    }
    if (m_spDeviceFile.get() == nullptr) {
        ERRLOG("Disk device file is nullptr.");
        return FAILED;
    }

    if (m_spDeviceFile->Read(offsetInBytes, bufSizeInBytes, buf) != DISK_DEVICE_OK) {
        ERRLOG("Read snapshot volume[%s] failed. offsetInBytes: %lld, bufSizeInBytes: %lld.",
               m_changeID.c_str(), offsetInBytes, bufSizeInBytes);
        return FAILED;
    }
    m_spDeviceFile->ReleaseBufferCache(offsetInBytes, bufSizeInBytes);
    if (m_backupType == AppProtect::BackupJobType::FULL_BACKUP) {
        if (memcmp(buf.get(), GetAllZeroDirtyRangeDataPtr().get(), DIRTY_RANGE_BLOCK_SIZE) == 0) {
            return DATA_ALL_ZERO_IGNORE_WRITE;
        }
    }
    return SUCCESS;
}

int32_t FusionStorageVolumeHandler::WriteBlocks(
    const uint64_t &offsetInBytes, uint64_t &bufSizeInBytes, std::shared_ptr<uint8_t[]> &buf)
{
    int32_t ret = SUCCESS;
    TP_START("TP_FusionStorageWriteFailed", 1, &ret);
    TP_END
    if (ret == FAILED) {
        return FAILED;
    }
    if (m_spDeviceFile.get() == nullptr) {
        ERRLOG("Disk device file is nullptr.");
        return FAILED;
    }

    if (m_spDeviceFile->Write(offsetInBytes, bufSizeInBytes, buf) != DISK_DEVICE_OK) {
        ERRLOG("Write volume[%s] failed. offsetInBytes: %lld, bufSizeInBytes: %lld.",
               m_volumeID.c_str(), offsetInBytes, bufSizeInBytes);
        return FAILED;
    }
    return SUCCESS;
}

uint64_t FusionStorageVolumeHandler::GetVolumeSize()
{
    return m_volSizeInBytes;
}

void FusionStorageVolumeHandler::DetachVolume(
    Json::Value &itemJson, const std::string &volumeName, std::string &attachedVolumes, int32_t &result)
{
    INFOLOG("Begin detach volume, volumeName: %s.", volumeName.c_str());
    std::string errMsg;
    int32_t iRet;

    if (m_apiMode == "VBS") {
        iRet = RetryOp(boost::bind(&FusionStorageApi::DetachVolume, m_spDSWareApi, volumeName, boost::ref(errMsg),
            m_cacheRepoHandler, m_cacheRepoPath + VIRT_PLUGIN_CACHE_ROOT),
            "DetachVolume");
    } else {
        iRet = RetryOp(
            boost::bind(&FusionStorageRestApiOperator::DetachVolume, m_spDSWareRestApi, volumeName, boost::ref(errMsg)),
            "DetachVolume");
    }

    if (iRet != DSWARE_API_OK) {
        ERRLOG("Detach bitmap volume data[%s] failed. Error is %s.", volumeName.c_str(), errMsg.c_str());
        attachedVolumes = volumeName;
        result = FAILED;
    } else {
        m_cleanFile->RemoveSpecificArrayElement(itemJson, FusionStorageDef::BITMAP_VOLUMES_ATTACHED, volumeName);
    }
}

void FusionStorageVolumeHandler::DeleteBitmapVolume(
    Json::Value &itemJson, std::string &createdBitmapVolume, std::string &createVolumes, int32_t &result)
{
    INFOLOG("Begin delete bitmap volume, volumeName: %s.", createdBitmapVolume.c_str());
    std::string errMsg;
    int32_t iRet;
    if (m_apiMode == "VBS") {
        iRet = RetryOp(
            boost::bind(&FusionStorageApi::DeleteBitmapVolume, m_spDSWareApi, createdBitmapVolume, boost::ref(errMsg)),
            "Delete Volume");
    } else {
        iRet = RetryOp(
            boost::bind(&FusionStorageRestApiOperator::DeleteBitmapVolume, m_spDSWareRestApi, createdBitmapVolume,
            boost::ref(errMsg)), "Delete Volume");
    }
    if (iRet != DSWARE_API_OK) {
        ERRLOG("Delete bitmap volume data[%s] failed. Error is %s.", createdBitmapVolume.c_str(), errMsg.c_str());
        createVolumes = createdBitmapVolume;
        result = FAILED;
    } else {
        m_cleanFile->RemoveSpecificArrayElement(
            itemJson, FusionStorageDef::BITMAP_VOLUMES_CREATED, createdBitmapVolume);
    }
}

int32_t FusionStorageVolumeHandler::CleanLeftoversForBackup(std::vector<std::string> &itemVecStr)
{
    Json::Value mountPointInfo;
    if (!Module::JsonHelper::JsonStringToJsonValue(itemVecStr[1], mountPointInfo)) {
        ERRLOG("Convert item string to json failed.");
        return FAILED;
    }

    int32_t result = SUCCESS;
    std::string attachedVolumes("None");
    std::string volumeName = mountPointInfo[FusionStorageDef::BITMAP_VOLUMES_ATTACHED][0].asString();
    std::string errMsg;
    if (!volumeName.empty()) {
        DetachVolume(mountPointInfo, volumeName, attachedVolumes, result);
    }
    std::string createVolumes("None");
    std::string createdBitmapVolume = mountPointInfo[FusionStorageDef::BITMAP_VOLUMES_CREATED][0].asString();
    if (!createdBitmapVolume.empty()) {
        DeleteBitmapVolume(mountPointInfo, createdBitmapVolume, createVolumes, result);
    }

    volumeName = mountPointInfo[FusionStorageDef::SNAPSHOT_VOLUMES_ATTACHED][0].asString();
    int32_t iRet;
    if (!volumeName.empty()) {
        if (m_apiMode == "VBS") {
            iRet = RetryOp(boost::bind(&FusionStorageApi::DetachVolume, m_spDSWareApi, volumeName, boost::ref(errMsg),
            m_cacheRepoHandler, m_cacheRepoPath + VIRT_PLUGIN_CACHE_ROOT),
            "DetachVolume");
        } else {
            iRet = RetryOp(
                boost::bind(
                    &FusionStorageRestApiOperator::DetachVolume, m_spDSWareRestApi, volumeName, boost::ref(errMsg)),
                "DetachVolume");
        }
        if (iRet != DSWARE_API_OK) {
            ERRLOG("Detach volume data[%s] failed. Error is %s.", volumeName.c_str(), errMsg.c_str());
            attachedVolumes = (attachedVolumes == "None") ? volumeName : attachedVolumes + ";\n" + volumeName;
            result = FAILED;
        } else {
            m_cleanFile->RemoveSpecificArrayElement(
                mountPointInfo, FusionStorageDef::SNAPSHOT_VOLUMES_ATTACHED, volumeName);
        }
    }

    if (result == SUCCESS) {
        // 挂载点都清理成功后，删除挂载点信息
        m_cleanFile->RemoveItem(itemVecStr[0]);
    }

    return result;
}

int32_t FusionStorageVolumeHandler::CleanLeftoversForRestore(std::vector<std::string> &itemVecStr)
{
    Json::Value mountPointInfo;
    if (!Module::JsonHelper::JsonStringToJsonValue(itemVecStr[1], mountPointInfo)) {
        ERRLOG("Convert item string to json failed.");
        return FAILED;
    }

    int32_t result = SUCCESS;
    std::string attachedVolumes("None");

    std::string volumeName = mountPointInfo[FusionStorageDef::BITMAP_VOLUMES_ATTACHED][0].asString();
    std::string errMsg;
    if (!volumeName.empty()) {
        DetachVolume(mountPointInfo, volumeName, attachedVolumes, result);
    }

    std::string createVolumes("None");
    std::string createdBitmapVolume = mountPointInfo[FusionStorageDef::BITMAP_VOLUMES_CREATED][0].asString();
    if (!createdBitmapVolume.empty()) {
        DeleteBitmapVolume(mountPointInfo, createdBitmapVolume, createVolumes, result);
    }

    volumeName = mountPointInfo[FusionStorageDef::DATA_VOLUMES_ATTACHED][0].asString();
    int32_t iRet;
    if (!volumeName.empty()) {
        if (m_apiMode == "VBS") {
            iRet = RetryOp(boost::bind(&FusionStorageApi::DetachVolume, m_spDSWareApi, volumeName, boost::ref(errMsg),
            m_cacheRepoHandler, m_cacheRepoPath + VIRT_PLUGIN_CACHE_ROOT),
            "DetachVolume");
        } else {
            iRet = RetryOp(
                boost::bind(
                    &FusionStorageRestApiOperator::DetachVolume, m_spDSWareRestApi, volumeName, boost::ref(errMsg)),
                "DetachVolume");
        }
        if (iRet != DSWARE_API_OK) {
            ERRLOG("Detach volume data[%s] failed. Error is %s.", volumeName.c_str(), errMsg.c_str());
            attachedVolumes = (attachedVolumes == "None") ? volumeName : attachedVolumes + ";\n" + volumeName;
            result = FAILED;
        } else {
            m_cleanFile->RemoveSpecificArrayElement(
                mountPointInfo, FusionStorageDef::DATA_VOLUMES_ATTACHED, volumeName);
        }
    }

    if (result == SUCCESS) {
        // 挂载点都清理成功后，删除挂载点信息
        m_cleanFile->RemoveItem(itemVecStr[0]);
    }

    return result;
}

int32_t FusionStorageVolumeHandler::CleanLeftovers()
{
    std::vector<std::string> items;
    int32_t iRet = m_cleanFile->GetAllItems(items);
    if (iRet != SUCCESS) {
        ERRLOG("Get all items failed.");
        return FAILED;
    }

    for (const auto &strItem : items) {
        // 清理逻辑：（1）清理本次缓存id的所有资源；（2）清理其余缓存id的未使用资源或者已超时资源
        if (strItem.find(m_dataPersistence->GetCachedUUID()) != std::string::npos ||
            (strItem.find(STATUS_IDLE) != std::string::npos || CheckMountPointIsExpire(strItem) == SUCCESS)) {
            INFOLOG("Begin clean item. strItem: %s", strItem.c_str());
            std::vector<std::string> itemVecStr;
            StrSplit(itemVecStr, strItem, '|');
            if (m_isBackup) {
                iRet = CleanLeftoversForBackup(itemVecStr);
            } else {
                iRet = CleanLeftoversForRestore(itemVecStr);
            }

            if (iRet != SUCCESS) {
                WARNLOG("Clean remain mount point failed. strItem: %s", strItem.c_str());
            }
        }
    }

    return SUCCESS;
}

int32_t FusionStorageVolumeHandler::RetryOp(boost::function<int()> fun, const std::string &opName)
{
    if (fun == nullptr) {
        ERRLOG("Operation[%s] failed.", opName.c_str());
        return FAILED;
    }

    int iRet = fun();
    if (iRet == DSWARE_API_OK) {
        return SUCCESS;
    }

    int tryTimes = 0;
    while (tryTimes < RETRY_TIMES) {
        tryTimes++;
        std::this_thread::sleep_for(std::chrono::seconds(RETRY_WAITING_TIME));

        iRet = fun();
        if (iRet == DSWARE_API_OK) {
            return SUCCESS;
        }

        WARNLOG("Operation[%s] failed. Retry %d time.", opName.c_str(), tryTimes);
    }

    ERRLOG("Operation[%s] failed.", opName.c_str());
    return iRet;
}

int32_t FusionStorageVolumeHandler::RegKeyForVolume(const std::string &diskDevicePath)
{
    DBGLOG("Start to set key for disk, disk device path: %s.", diskDevicePath.c_str());

    // reg_fs.sh reg_key [diskDevicePath]
    std::string agentHomedir = Module::EnvVarManager::GetInstance()->GetAgentHomePath();
    std::string regPath = agentHomedir + REG_FS;
    std::vector<std::string> resultvec;
    resultvec.clear();
    std::unordered_set<std::string> pathWhiteList;
    pathWhiteList.insert(diskDevicePath);
    std::vector<Module::CmdParam> cmdParam {
        Module::CmdParam(Module::COMMON_CMD_NAME, "sudo"),
        Module::CmdParam(Module::PATH_PARAM, regPath),
        "reg",
        Module::CmdParam(Module::PATH_PARAM, diskDevicePath)
    };
    int runResult = Utils::CallAgentExecCmd(cmdParam, resultvec);
    if (runResult != 0) {
        ERRLOG("Set key for disk: %s failed.", diskDevicePath.c_str());
        return FAILED;
    }

    INFOLOG("Set key for disk %s successfully.", diskDevicePath.c_str());
    return SUCCESS;
}

int32_t FusionStorageVolumeHandler::UnRegKeyForVolume(const std::string &diskDevicePath)
{
    DBGLOG("Start to remove key for disk, disk device path: %s.", diskDevicePath.c_str());

    // reg_fs.sh unreg_key [diskDevicePath]
    std::string agentHomedir = Module::EnvVarManager::GetInstance()->GetAgentHomePath();
    std::string regPath = agentHomedir + REG_FS;
    std::vector<std::string> resultvec;
    resultvec.clear();

    std::unordered_set<std::string> pathWhiteList;
    pathWhiteList.insert(diskDevicePath);
    std::vector<Module::CmdParam> cmdParam {
        Module::CmdParam(Module::COMMON_CMD_NAME, "sudo"),
        Module::CmdParam(Module::PATH_PARAM, regPath),
        "unreg",
        Module::CmdParam(Module::PATH_PARAM, diskDevicePath)
    };
    int runResult = Utils::CallAgentExecCmd(cmdParam, resultvec);
    if (runResult != 0) {
        ERRLOG("Remove key for disk: %s failed.", diskDevicePath.c_str());
        return FAILED;
    }

    INFOLOG("Remove key for disk %s successfully.", diskDevicePath.c_str());
    return SUCCESS;
}

void FusionStorageVolumeHandler::ReleaseBufferCache(const uint64_t &offsetInBytes, const uint64_t &readLen)
{
    DBGLOG("FusionStorageVolumeHandler ReleaseBufferCache.");
    m_spDeviceFile->ReleaseBufferCache(offsetInBytes, readLen);
}

void FusionStorageVolumeHandler::StrSplit(std::vector<std::string> &vecTokens, const std::string &strText, char cSep)
{
    std::string::size_type start = 0;
    std::string::size_type end;
    while ((end = strText.find(cSep, start)) != std::string::npos) {
        vecTokens.push_back(strText.substr(start, end - start));
        start = end + 1;
    }

    vecTokens.push_back(strText.substr(start));
}

int32_t FusionStorageVolumeHandler::CheckMountPointIsExpire(const std::string &strItem)
{
    int pos = strItem.find_last_of('|') + 1;
    std::string itemTimeStamp = strItem.substr(pos, strItem.size() - pos);
    // 获取当前时间戳, 单位: s
    uint64_t curTimeStamp =
            std::chrono::system_clock::now().time_since_epoch().count() / std::chrono::system_clock::period::den;

    uint64_t timeStampDiff = curTimeStamp - atoll(itemTimeStamp.data());
    uint64_t expireTime = Module::ConfigReader::getInt(
        std::string("HcsConfig"), std::string("LongestTimeBackUpOneVolume"));
    if (timeStampDiff > expireTime) {
        // 挂载点过期，需要清理
        return SUCCESS;
    }

    return FAILED;
}

int32_t FusionStorageVolumeHandler::Flush()
{
    return m_spDeviceFile->Flush() == DISK_DEVICE_OK ? Module::SUCCESS : Module::FAILED;
}

std::shared_ptr<JobHandle> FusionStorageVolumeHandler::GetJobHandleInTask()
{
    return m_jobHandle;
}

int32_t FusionStorageVolumeHandler::QueryStoragePoolUsedRate(double &usedCapacityRate)
{
    std::string errDesc;
    if (m_apiMode == "VBS") {
        if (m_spDSWareApi->QueryStoragePoolUsedRate(usedCapacityRate) != SUCCESS) {
            ERRLOG("Fusion storage query storage pool used rate failed.");
            return FAILED;
        }
    } else {
        if (m_spDSWareRestApi->QueryStoragePoolUsedRate(usedCapacityRate) != SUCCESS) {
            ERRLOG("Fusion storage query storage pool used rate failed.");
            return FAILED;
        }
    }
    INFOLOG("Fusion storage query storage pool used rate success.");
    return SUCCESS;
}

VIRT_PLUGIN_NAMESPACE_END
